import requests
import json
import Common

url = 'https://www.baidu.com'

response = requests.get(url)
print(response)

res = Common.get_robots_txt(url)
print(res)
